# 案例1: 中国大学排名(不能用selenium拿数据)
# url: https://www.shanghairanking.cn/rankings/bcur/202111
# 下载该网站2015-2022年的中国大学排名数据, 每一年所有学校的数据都要,包含的字段有: 排名, 学校名称, 省市, 类型, 总分, 办学层次(生源质量)
# 注意:
# 2015-2019年的拿排名, 学校名称, 省市, 类型, 总分, 生源质量
# 2020-2022年的拿排名, 学校名称, 省市, 类型, 总分, 办学层次

import requests
from lxml import etree

def China_Uty(year):
    url = f"https://www.shanghairanking.cn/rankings/bcur/{year}"
    headers = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.51',
        'cookie': 'Hm_lvt_af1fda4748dacbd3ee2e3a69c3496570=1679840251,1679923824,1680081651,1680355874; Hm_lpvt_af1fda4748dacbd3ee2e3a69c3496570=1680355874'
    }
    r = requests.get(url, headers=headers).content.decode(encoding='utf-8')
    # print(r)
    ret = etree.HTML(r)
    tr_list = ret.xpath('//tbody//tr')
    # print(tr_list)

    for tr in tr_list:
        # 排名
        ranking = tr.xpath('.//td[1]//div[@data-v-3fe7d390]/text()')[0]
        ranking = str(ranking).strip()

        # 学校
        school = tr.xpath('.//td[2]//div[2]/div/div/div/a/text()')[0]
        school = str(school).strip()
        # 省市
        province = tr.xpath('.//td[3]/text()')[0]
        province =str(province).strip()
        # 类型
        mold = tr.xpath('.//td[4]/text()')[0]
        mold = str(mold).strip()
        # 总分
        score = tr.xpath('.//td[5]/text()')[0]
        score = str(score).strip()
        # 生源质量
        # biogen = tr.xpath('.//td[6]/text()')[0]
        # print(ranking+school+province+mold+score+" ")
        print(ranking,school,province,mold,score)
        # res = ranking+school+province+mold+score+" "

        # f.write(f'{res}')
        f.write(f'{ranking},{school},{province},{mold},{score}\n')


if __name__ == '__main__':
    with open('中国大学.csv', 'a', encoding='utf-8-sig') as f:
        f.write("排名,学校,省市,类型,总分\n")
        # for year in range(2015,2020):
        year = 2015

        while year<=2020:
            China_Uty(year)
            year+=1